{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip install -q openai==0.28.0\n",
    "# !pip install -q langchain\n",
    "# !pip install -q guardrails-ai\n",
    "# !pip install -q faiss-cpu\n",
    "# !pip install -q pypdf\n",
    "# !pip install -q python-dotenv\n",
    "# !pip install -q datasets\n",
    "# !pip install -q huggingface_hub\n",
    "# !pip install tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from dotenv import load_dotenv\n",
    "import json\n",
    "\n",
    "from langchain.chains import RetrievalQA\n",
    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
    "from langchain.vectorstores import FAISS\n",
    "from langchain.document_loaders import PyPDFDirectoryLoader\n",
    "from langchain.chat_models import ChatOpenAI\n",
    "from langchain.schema import HumanMessage, AIMessage,  SystemMessage\n",
    "from langchain.document_loaders import YoutubeLoader\n",
    "\n",
    "\n",
    "from rich import print\n",
    "\n",
    "#Guardrails\n",
    "# from langchain.output_parsers import GuardrailsOutputParser\n",
    "# from langchain.prompts import PromptTemplate\n",
    "# from langchain.llms import OpenAI"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Name of Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_name = \"Dataset name\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Environment Varible"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from openai import OpenAI\n",
    "client = OpenAI()\n",
    "from dotenv import load_dotenv\n",
    "\n",
    "load_dotenv()\n",
    "import os\n",
    "\n",
    "openai_api_key = os.getenv(\"OPENAI_API_KEY\")\n",
    "client = OpenAI(\n",
    "    # This is the default and can be omitted\n",
    "    # api_key = openai_api_key,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "huggingface_api_key = os.environ.get(\"HUGGINGFACE_API_KEY\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Projects\\CognitiveLab\\clients\\gratitude\\taogpt7B\\taogpt\\Lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "from huggingface_hub import HfApi\n",
    "from datasets import load_dataset\n",
    "api = HfApi(token=huggingface_api_key)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading the Document"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# loader = PyPDFDirectoryLoader(\"/content/sample_data/Data/\")\n",
    "# loader = PyPDFDirectoryLoader(\"../cyber\")\n",
    "loader = PyPDFDirectoryLoader(\"../data\")\n",
    "data = loader.load()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(data[0])\n",
    "\n",
    "print(len(data))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading Transcript from Youtube"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "list_of_urls = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from rich import print\n",
    "\n",
    "# overall_data = []\n",
    "\n",
    "for url in list_of_urls:\n",
    "    loader = YoutubeLoader.from_youtube_url(url, add_video_info=True)\n",
    "    youtube_data = loader.load()\n",
    "    data.append(youtube_data[0])\n",
    "print(len(data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_name = \"Extracted_data.txt\"\n",
    "\n",
    "# Open the file in write mode\n",
    "with open(file_name, \"w\",encoding=\"utf-8\") as file:\n",
    "    for item in data:\n",
    "        # Extract the \"page_content\" attribute from each item in the list\n",
    "        page_content = item.page_content\n",
    "        # Write the page_content to the file followed by a newline\n",
    "        file.write(page_content + \"\\n\")\n",
    "print(f\"Data has been written to {file_name}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Chunking the Documents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Step 05: Split the Extracted Data into Text Chunks\n",
    "# text_splitter = RecursiveCharacterTextSplitter(chunk_size=1500, chunk_overlap=500)\n",
    "text_splitter = RecursiveCharacterTextSplitter(chunk_size=1500, chunk_overlap=500)\n",
    "text_chunks = text_splitter.split_documents(data)\n",
    "print(\"Lenth of the whole documentation is:\",len(text_chunks))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prompt template\n",
    "\n",
    "[INPUT TEXT]\n",
    "\n",
    "[CONTEXT ]\n",
    "\n",
    "[REQUEST FOR Q+A]\n",
    "\n",
    "[RESPONSE SAMPLE]\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Getting Context of the Data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generating QnA \n",
    "\n",
    "- *prompt* : provide {questions_per_chunk} question and answer pairs base on the text above , The Question must begin with \\n\"In the context of ...\\\".The answer borrow, verbatim, from the text above. In providing each question consider that the reader does not see of have access to any of the other questions from context. Vary the style and formate fo quesitons. Respond in plain test on a new line for each question and answer. Do not include Do no include qestion numbers, Here is an exmaple of two question and answer paids:\\n\\n {tain_sample}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "json_response_format = [\n",
    "                {\n",
    "                    \"question\": \"In the context of ...\",\n",
    "                    \"answer\": \"...\"\n",
    "                },\n",
    "                {\n",
    "                    \"question\": \"In the context of ...\",\n",
    "                    \"answer\": \"...\"\n",
    "                },\n",
    "                {\n",
    "                    \"question\": \"In the context of ...\",\n",
    "                    \"answer\": \"...\"\n",
    "                }\n",
    "            ]\n",
    "\n",
    "import json\n",
    "\n",
    "def save_to_jsonl(dataset_name, question_answer_list):\n",
    "    \"\"\"\n",
    "    Save question-answer pairs with metadata to a JSONL file.\n",
    "\n",
    "    Parameters:\n",
    "        - dataset_name (str): The name of the dataset.\n",
    "        - question_answer_list (list): List of dictionaries containing question, answer, and metadata.\n",
    "    \"\"\"\n",
    "    file_name = f\"../data/{dataset_name}.jsonl\"\n",
    "\n",
    "    with open(file_name, \"a\", encoding=\"utf-8\") as file:\n",
    "        for qa in question_answer_list:\n",
    "            # Combine question, answer, and metadata into a dictionary\n",
    "            entry = {\n",
    "                \"question\": qa[\"question\"],\n",
    "                \"answer\": qa[\"answer\"],\n",
    "                \"content\": qa.get(\"content\", \"\"),\n",
    "            }\n",
    "            # Write the dictionary to the JSONL file\n",
    "            file.write(json.dumps(entry, ensure_ascii=False) + \"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_text(text):\n",
    "    # Short Response\n",
    "    short_response = client.chat.completions.create(\n",
    "        model=\"gpt-3.5-turbo-1106\",\n",
    "        temperature=0.3,\n",
    "        response_format={\"type\": \"json_object\"},\n",
    "        messages=[\n",
    "            {\n",
    "                \"role\": \"system\",\n",
    "                \"content\": f\"\"\"You are an expert at qurating/generating questions and answers from a given piece of text.\n",
    "                            The questions and answers you generate are unique from one another and are not repeated.\n",
    "                            You always respond in the following json format ```question_answer:{json_response_format}```\"\"\"\n",
    "            },\n",
    "            {\n",
    "                \"role\": \"user\",\n",
    "                # you should add the summary and modify the prompt to your liking\n",
    "                \"content\": f\"\"\"given the context which is about *summary of the document* \n",
    "                \\n\n",
    "                {text.page_content} \n",
    "                \\n\n",
    "                provide 10 question and answer pairs base on the text above , \n",
    "                The Question must begin with \"In the context of ...\\\".The answer borrow, verbatim, from the text above. \n",
    "                In providing each question consider that the reader does not see or have access to any of the other questions from context. \n",
    "                Vary the style and format of questions. Let the answers be descriptive around 100 to 200 words\n",
    "                \"\"\"\n",
    "                # Respond in only JSON following this format and nothing else {json_response_format}\n",
    "            }\n",
    "        ]\n",
    "    )\n",
    "\n",
    "    try:\n",
    "        short_response_output = eval(short_response.choices[0].message.content)\n",
    "\n",
    "        # Check if the format is correct and meets the criteria\n",
    "        if (\n",
    "            isinstance(short_response_output, dict)\n",
    "            and \"question_answer\" in short_response_output\n",
    "            and isinstance(short_response_output[\"question_answer\"], list)\n",
    "            and len(short_response_output[\"question_answer\"]) >= 9\n",
    "            and all(\n",
    "                isinstance(qa, dict)\n",
    "                and \"question\" in qa\n",
    "                and \"answer\" in qa\n",
    "                for qa in short_response_output[\"question_answer\"]\n",
    "            )\n",
    "        ):\n",
    "            # Add metadata to each question-answer pair\n",
    "            for qa in short_response_output[\"question_answer\"]:\n",
    "                qa[\"content\"] = f\"{text.page_content}\"\n",
    "                qa[\"metadata\"] = text.metadata\n",
    "\n",
    "            # print(\"Short response format is correct.\")\n",
    "        else:\n",
    "            print(\"Short response format is incorrect. Running the query again.\")\n",
    "            \n",
    "\n",
    "    except Exception as e:\n",
    "        print(\"Error in short_response_output\", e)\n",
    "\n",
    "    # Long Response\n",
    "    long_response = client.chat.completions.create(\n",
    "        model=\"gpt-3.5-turbo-1106\",\n",
    "        temperature=0.3,\n",
    "        response_format={\"type\": \"json_object\"},\n",
    "        messages=[\n",
    "            {\n",
    "                \"role\": \"system\",\n",
    "                \"content\": f\"\"\"You are an expert at qurating/generating questions and answers from a given piece of text.\n",
    "                            The questions and answers you generate are unique from one another and are not repeated.\n",
    "                            You always respond in the following json format ```question_answer:{json_response_format}```\"\"\"\n",
    "            },\n",
    "            {\n",
    "                \"role\": \"user\",\n",
    "                \"content\": f\"\"\"given the context which is about Tao Science which is written by \"Rulin Xiu\" and \"Zhi Gang Sha\"\n",
    "                \\n\n",
    "                {text.page_content} \n",
    "                \\n\n",
    "                provide 5 question and answer pairs base on the text above , \n",
    "                The Question must begin with \"In the context of...\\\".The answer borrow, verbatim, from the text above. \n",
    "                In providing each question consider that the reader does not see or have access to any of the other questions from context. \n",
    "                Vary the style and format of questions. Let the answers be descriptive and lengthy.\n",
    "                The answer should at least be 1000 words\n",
    "                \"\"\"\n",
    "                # Respond in only JSON following this format and nothing else {json_response_format}\n",
    "            }\n",
    "        ]\n",
    "    )\n",
    "\n",
    "    try:\n",
    "        long_response_output = eval(long_response.choices[0].message.content)\n",
    "\n",
    "        # Check if the format is correct and meets the criteria\n",
    "        if (\n",
    "            isinstance(long_response_output, dict)\n",
    "            and \"question_answer\" in long_response_output\n",
    "            and isinstance(long_response_output[\"question_answer\"], list)\n",
    "            and len(long_response_output[\"question_answer\"]) >= 4\n",
    "            and all(\n",
    "                isinstance(qa, dict)\n",
    "                and \"question\" in qa\n",
    "                and \"answer\" in qa\n",
    "                for qa in long_response_output[\"question_answer\"]\n",
    "            )\n",
    "        ):\n",
    "            # Add metadata to each question-answer pair\n",
    "            for qa in long_response_output[\"question_answer\"]:\n",
    "                qa[\"content\"] = f\"{text.page_content}\"\n",
    "                qa[\"metadata\"] = text.metadata\n",
    "\n",
    "            # print(\"Long response format is correct.\")\n",
    "        else:\n",
    "            print(\"Long response format is incorrect. Running the query again.\")\n",
    "            \n",
    "\n",
    "    except Exception as e:\n",
    "        print(\"Error in long_response_output\", e)\n",
    "    save_to_jsonl(dataset_name, short_response_output[\"question_answer\"])\n",
    "    save_to_jsonl(dataset_name, long_response_output[\"question_answer\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Lets start synthetically generating the prompts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import concurrent.futures\n",
    "from tqdm import tqdm\n",
    "\n",
    "with concurrent.futures.ThreadPoolExecutor() as executor:\n",
    "    # Submit each text for processing\n",
    "    futures = [executor.submit(process_text, text) for text in text_chunks]\n",
    "\n",
    "    # Use tqdm to create a progress bar\n",
    "    for _ in tqdm(concurrent.futures.as_completed(futures), total=len(futures), desc=\"Processing\"):\n",
    "        pass  \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## HuggingFace"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['question', 'answer', 'metadata', 'content'],\n",
       "        num_rows: 6569\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset = load_dataset('json', data_files=\"./stringified_TaoGPT-v2.jsonl\")\n",
    "dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Creating parquet from Arrow format: 100%|██████████| 7/7 [00:00<00:00, 143.84ba/s]\n",
      "Uploading the dataset shards: 100%|██████████| 1/1 [00:02<00:00,  2.60s/it]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "CommitInfo(commit_url='https://huggingface.co/datasets/agency888/TaoGPT-v2/commit/fd5e95ec764d1e04b8eacfc1d42bc9217d2956a3', commit_message='Upload dataset', commit_description='', oid='fd5e95ec764d1e04b8eacfc1d42bc9217d2956a3', pr_url=None, pr_revision=None, pr_num=None)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# dataset.push_to_hub(\"Dataset name\")\n",
    "dataset.push_to_hub(\"account/datastname\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Pushing Dataset.json\n",
    "api.upload_file(\n",
    "    path_or_fileobj=\"./dataset.json\",\n",
    "    path_in_repo=\"dataset.json\",\n",
    "    repo_id=\"account/datastname\",\n",
    "    repo_type=\"dataset\",\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Pushing Dataset Readme\n",
    "api.upload_file(\n",
    "    path_or_fileobj=\"./README.md\",\n",
    "    path_in_repo=\"README.md\",\n",
    "    repo_id=\"account/datastname\",\n",
    "    repo_type=\"dataset\",\n",
    ")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
